package com.zhanggm.s3.house.webmagic;

import cn.hutool.core.lang.copier.SrcToDestCopier;
import org.jsoup.Jsoup;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class AnJuKeSpider implements PageProcessor {

    Site site = Site.me().setTimeOut(10000).setRetryTimes(5).setSleepTime(1000)
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0");

    @Override
    public void process(Page page) {
        //爬取的页面详情
        Html html = page.getHtml();
        //目标链接
        List<String> targetRequestList = html.xpath("//div[@class='multi-page']").links().all();
        page.addTargetRequests(targetRequestList);
        //解析的数据集，多条数据
        List<Map<String, Object>> pageData = new ArrayList<>();
        //获取当前页，房源列表html节点
        List<Selectable> nodes = html.xpath("//div[@class='zu-itemmod']").nodes();
        //遍历解析房源信息
        String fromUrl = page.getUrl().get();
        nodes.forEach(node->{
            Map<String, Object> houseMap = new HashMap<>();
            houseMap.put("fromUrl", fromUrl);
            houseMap.put("img", node.$("img", "lazy_src").get());
            houseMap.put("title", node.xpath("//b[@class='strongbox']/text()").get());
            Selectable detail = node.xpath("//p[@class='details-item tag']/allText()");
            houseMap.put("detailItemTag", detail.get());
            houseMap.put("detailItem", node.xpath("//address[@class='details-item']/text()").get().trim());
            houseMap.put("detailItemBotTag", node.xpath("//p[@class='details-item bot-tag']/allText()").get());
            houseMap.put("price", node.xpath("//div[@class='zu-side']/allText()").get());
            pageData.add(houseMap);
            System.out.println(houseMap);
        });
        page.putField("pageData", pageData);
    }

    @Override
    public Site getSite() {
        return site;
    }


    public static void main(String[] args) {
        Spider.create(new AnJuKeSpider())
                .addUrl("")
                .addPipeline(new Pipeline() {
                    @Override
                    public void process(ResultItems resultItems, Task task) {
                        List<Map<String, Object>> pageData = (List<Map<String, Object>>)resultItems.getAll().get("pageData");
                        System.out.println(pageData);
                    }
                })
                .addPipeline(new FilePipeline("d:\\webmagic\\"))
                .run();
    }
}
